import requests
import re
import time
from bs4 import BeautifulSoup
from random import randint

titles = set()
authors = set()
replies = set()
levels = set()
timedates = set()
sum = []
urlgroup=[]
urlbase=[]

def tieba():
    ll_re = re.compile(r'(.*(?i)love.*live.*sunshine.*)', re.I)
    url_re=re.compile(r'(/p/\d{10})',re.I)
    time_re = re.compile('(\d{1}\D)' , re.I)
    i = 0

    print('process', i, 'page...')
    url = 'http://tieba.baidu.com/f?kw=lovelivesunshine&ie=utf-8&pn=' + str(i)

    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html5lib')
    question_links = soup.select('a[href]')


    # check error page
    if soup.find('div', class_='error') is not None:
        print('end.')

    for link in question_links:
        title = link.text.strip() + '\n'
        if ll_re.match(title) is not None and title not in titles:
            if title not in titles:
                if url_re.match(link["href"]) is not None and link["href"] not in urlbase:
                # print('matched:', title[:-1])
                    titles.add(title)
                    headers = {
                        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36'}
                    new_urls = 'http://tieba.baidu.com%s' % link['href']
                    urlbase.append(link['href'])
                    try:
                        r_new = requests.get(new_urls , headers=headers)
                        soup = BeautifulSoup(r_new.text , 'html5lib')
                        author = soup.find("li" , class_="d_name").select('a[alog-group]')
                        reply = soup.find("li" , class_="l_reply_num").select('span[style]')
                        level = soup.find('div' , class_="d_badge_lv").string
                        timedata = soup.find_all('span' , class_="tail-info")
                        if time_re.match(timedata[1].string) is not None:
                            timedates.add(timedata[2].string)
                        else:
                            timedates.add(timedata[1].string)
                        levels.add(level)
                        for author_name in author:
                            authors.add(author_name.string)
                        for num in reply:
                            replies.add(num.string)
                    except:
                        pass
    length=len(titles)
    sum.extend(titles)
    sum.extend(levels)
    sum.extend(timedates)
    sum.extend(authors)
    sum.extend(replies)
    for num in range(length):
        for n in range(5):
            print sum[n * length+num]

tieba()